#
# Author: yizhigopher
# Description: DQN深度学习神经网络
# File: dqn.py
# Date: 2023/11/16 15:15:17
#
import numpy as np
import torch.nn as nn
import torch

epsilon = 0.1  # epsilon-greedy algorithm
alpha = 0.1
gamma = 0.98  # discount factor
target_update = 10  # how often update Q-net
learning_rate = 2e-3  # optimizer learning rate
device = "cuda" if torch.cuda.is_available() else "cpu"
hidden_dim = 128
buffer_size = 10000


class DQN(nn.Module):
    """
    先尝试用用这个, 之后试试DDPG算法
    """

    def __init__(self, state_dim, hidden_dim, action_dim):
        super(DQN, self).__init__()
        # define the simplest neural net
        self.layer = nn.Sequential(
            nn.Linear(state_dim, hidden_dim),
            nn.ReLU(),
            nn.Linear(hidden_dim, hidden_dim),
            nn.ReLU(),
            nn.Linear(hidden_dim, action_dim)
        )

    def forward(self, x):
        return self.layer(x)


class DQLearning:

    def __init__(self, state_dim: int, hidden_dim: int, action_dim: int):
        self.state_dim = state_dim
        self.action_dim = action_dim
        self.Qnet = DQN(state_dim, hidden_dim, action_dim).to(device)
        self.Target = DQN(state_dim, hidden_dim, action_dim).to(device)
        self.optimizer = torch.optim.Adam(
            self.Qnet.parameters(), lr=learning_rate)
        self.count = 0
        self.criterion = nn.MSELoss()

    def take_action(self, state):
        if np.random.random() < epsilon:
            actions = []
            for i in range(self.state_dim):
                actions.append(np.random.randint(15000))
            return np.random.randint(self.action_dim)
        else:
            state = torch.tensor([state], dtype=torch.float).to(device)
            return self.Qnet(state).argmax().item()

    def update(self, transition_dict):
        states = torch.tensor(transition_dict['states'], dtype=torch.float).to(device)
        actions = torch.tensor(transition_dict['actions']).view(-1, 1).to(device)
        rewards = torch.tensor(transition_dict['rewards'], dtype=torch.float).view(-1, 1).to(device)
        next_states = torch.tensor(transition_dict['next_states'], dtype=torch.float).to(device)
        dones = torch.tensor(transition_dict['dones'], dtype=torch.float).view(-1, 1).to(device)

        q_values = self.Qnet(states).gather(1, actions)
        max_next_q_values = self.target(next_states).max(1)[0].view(-1, 1)
        q_targets = rewards + gamma * max_next_q_values * (1 - dones)
        dqn_loss = self.criterion(q_values, q_targets)
        self.optimizer.zero_grad()
        dqn_loss.backward()
        self.optimizer.step()

        if self.count % target_update == 0:
            self.target.load_state_dict(self.Qnet.state_dict())
        self.count += 1
